16. 缺失值处理：长三角上市公司验证（英国零售对照）

from pathlib import Path  # 导入Path以兼容Linux与Windows数据根目录
import pandas as pd  # 导入Pandas用于读取和连接本地财报

linux_root = Path('/home/ubuntu/r2_data_mount/data')  # 指定Linux课程数据根目录
windows_root = Path(r'C:\qiufei\data')  # 指定Windows课程数据根目录
data_root = linux_root if linux_root.exists() else windows_root  # 选择当前平台可用根目录
financial_path = data_root / 'stock' / 'financial_statement.h5'  # 定位财务报表快照
basic_path = data_root / 'stock' / 'stock_basic_data.h5'  # 定位公司地区基础表

if financial_path.exists() and basic_path.exists():  # 仅在课程数据挂载存在时执行实证
    financials = pd.read_hdf(financial_path, key='financial_data', columns=['order_book_id','quarter','operating_revenue','net_profit']).rename(columns={'order_book_id':'ts_code','quarter':'end_date','operating_revenue':'total_revenue','net_profit':'n_income'})  # 只读取必需财报字段并标准化
    companies = pd.read_hdf(basic_path, key='stock_basic_info', columns=['order_book_id','province']).rename(columns={'order_book_id':'ts_code','province':'area'})  # 只读取地区筛选字段并标准化
    financials['end_date'] = pd.PeriodIndex(financials['end_date'].str.upper(), freq='Q').end_time.normalize()  # 将季度键转换为报告期末日期
    companies['area'] = companies['area'].astype('string').str.replace(r'[省市]$','',regex=True)  # 将上海市、江苏省等标签统一为省市简称
    region_column = next(column for column in ['province', 'area', '地区'] if column in companies.columns)  # 识别地区字段
    target_regions = ['上海', '江苏', '浙江', '安徽']  # 定义长三角四省市口径
    regional_codes = companies.loc[companies[region_column].isin(target_regions), 'ts_code']  # 筛选长三角公司代码
    regional_financials = financials.loc[financials['ts_code'].isin(regional_codes) & financials['end_date'].between('2019-01-01', '2025-12-31')]  # 同时落实地区与2019—2025期间口径
    if regional_codes.empty or regional_financials.empty: raise ValueError('输入文件、字段、样本量或数值不符合当前分析要求，请按本页说明检查')  # 空地区或空期间不得发布NaN依据
    quality_report = regional_financials[['total_revenue', 'n_income']].isna().mean().rename('missing_rate')  # 计算关键字段缺失率
    if quality_report.empty or not quality_report.notna().all(): raise ValueError('输入文件、字段、样本量或数值不符合当前分析要求，请按本页说明检查')  # 缺失率必须非空且有限
    print(f'样本量：{len(regional_financials)}；报告期：{regional_financials["end_date"].min()} 至 {regional_financials["end_date"].max()}')  # 输出样本边界供核对
    print(quality_report.to_frame())  # 输出便于核对的数据质量结果
else:  # 其他运行环境未提供课程数据时保持明确提示
    print(f'本地数据挂载不可用：{data_root}')  # 仅在路径确实不可用时记录原因

import pandas as pd
import numpy as np

# 6家零售门店（A-F）的日销售数据
data = {
    'Store': ['A', 'B', 'C', 'D', 'E', 'F'],
    # 门店B和E的销售额缺失（系统故障或未营业）
    'Sales': [100, np.nan, 150, 200, np.nan, 180],
    # 门店C和F的利润缺失（成本数据未录入）
    'Profit': [10, 15, np.nan, 20, 25, np.nan],
    # 门店D的客户数缺失（客流计数器故障）
    'Customers': [50, 75, 100, np.nan, 80, 95]
}
df = pd.DataFrame(data)
print('原始数据:')
print(df)

import pandas as pd
import numpy as np

np.random.seed(42)
n_rows = 100
data_vis = pd.DataFrame({
    'A': np.random.rand(n_rows),
    'B': np.random.rand(n_rows),
    'C': np.random.rand(n_rows),
    'D': np.random.rand(n_rows)
})

# 场景1：变量A的10%随机缺失（MCAR）
mask_a = np.random.rand(n_rows) < 0.1
data_vis.loc[mask_a, 'A'] = np.nan

# 场景2：变量B的条件缺失（MAR：当A<0.5时B缺失）
data_vis.loc[data_vis['A'] < 0.5, 'B'] = np.nan

# 场景3：变量C的15%随机缺失
mask_c = np.random.rand(n_rows) < 0.15
data_vis.loc[mask_c, 'C'] = np.nan

# 场景4：变量D无缺失（对照组）
missing_matrix = data_vis.isnull()
print('各变量缺失值数量:')
print(missing_matrix.sum())

import pandas as pd
import numpy as np

data = {
    'Store': ['A', 'B', 'C', 'D', 'E'],
    'Sales': [100, np.nan, 150, 200, np.nan],
    'Profit': [10, 15, np.nan, 20, 25]
}
df = pd.DataFrame(data)
print('原始数据:')
print(df)
print()

# how='any'：只要行中有任何一个缺失值，就删除该行
df_drop_any = df.dropna(how='any')
print('删除含任何缺失值的行(how="any"):')
print(df_drop_any)

import pandas as pd
import numpy as np

# 10天的价格序列，含3个缺失值
dates = pd.date_range('2024-01-01', periods=10)
prices = [100.5, np.nan, 101.2, 100.8, np.nan, np.nan, 102.5, 103.0, np.nan, 104.2]
df_prices = pd.DataFrame({'Date': dates, 'Price': prices})
print('原始价格数据:')
print(df_prices)

# method='ffill'：用前一个有效值填充
df_ffill = df_prices.copy()
df_ffill['Price'] = df_ffill['Price'].fillna(method='ffill')
print('前向填充(用前一个值填充):')
print(df_ffill[['Date', 'Price']].head(4))

# method='bfill'：用后一个有效值填充
df_bfill = df_prices.copy()
df_bfill['Price'] = df_bfill['Price'].fillna(method='bfill')
print('后向填充(用后一个值填充):')
print(df_bfill[['Date', 'Price']].head(4))

# interpolate(method='linear')：在两个已知点之间线性插值
df_linear = df_prices.copy()
df_linear['Price'] = df_linear['Price'].interpolate(method='linear')
print('线性插值:')
print(df_linear[['Date', 'Price']].head(4))

import pandas as pd
import numpy as np

np.random.seed(42)
dates = pd.date_range('2024-01-01', periods=50)
prices = 100 + np.random.randn(50).cumsum()
prices[10:15] = np.nan  # 模拟连续5天停牌
prices[30:32] = np.nan  # 模拟连续2天缺失
prices[45] = np.nan      # 模拟单日缺失

df = pd.DataFrame({'Date': dates, 'Price': prices})
df.set_index('Date', inplace=True)

# 策略1：时间插值（考虑日期间隔）
df_time = df.interpolate(method='time')

# 策略2：多项式插值（二阶）
df_poly = df.interpolate(method='polynomial', order=2)

interpolation_check = pd.concat([df_time.rename(columns={'Price': '时间插值'}), df_poly.rename(columns={'Price': '多项式插值'})], axis=1).iloc[[9, 10, 14, 15]]  # 选取缺口边界行形成便于核对摘要
print('连续缺口边界的两种插值结果:')  # 标明摘要对应的业务检查点
print(interpolation_check)  # 输出缺口前后与缺口内部的代表性结果

import pandas as pd
import numpy as np

stocks = pd.DataFrame({
    'Date': list(pd.date_range('2024-01-01', periods=5)) * 2,
    'Stock': ['A'] * 5 + ['B'] * 5,
    'Price': [100, 101, np.nan, 103, 104,
              50, np.nan, 52, 53, np.nan]
})
missing_before = stocks.groupby('Stock')['Price'].apply(lambda series: series.isna().sum())  # 记录各股票填充前缺失数

# 按股票代码分组，分别前向填充
stocks['Price'] = stocks.groupby('Stock')['Price'].fillna(method='ffill')
missing_after = stocks.groupby('Stock')['Price'].apply(lambda series: series.isna().sum())  # 计算各股票填充后缺失数
group_fill_check = pd.DataFrame({'填充前缺失数': missing_before, '填充后缺失数': missing_after})  # 汇总分组填充的便于核对变化
print('分组填充质量摘要:')  # 标明输出用于检查组间隔离与缺失变化
print(group_fill_check)  # 输出每只股票填充前后的缺失数

import pandas as pd
import numpy as np

# 确定性课堂示例：只用于演示字段结构与缺失处理，不代表任何企业或市场
df_retail = pd.DataFrame({
    'InvoiceID': ['FIX-01','FIX-02','FIX-03','FIX-04','FIX-05','FIX-06'],
    'CustomerID': ['C01','C02',np.nan,'C02','C03','C03'],
    'InvoiceDate': pd.date_range('2023-01-01', periods=6),
    'Quantity': [2,4,3,np.nan,5,1],
    'UnitPrice': [10.0,12.0,8.0,12.0,np.nan,9.0],
    'is_observed': [True] * 6
})
retail_quality = pd.DataFrame({'缺失数': df_retail.isnull().sum(), '缺失率(%)': df_retail.isnull().mean() * 100})  # 汇总字段级缺失规模与比例
print('课堂示例（非业务观测）形状:', df_retail.shape)  # 输出教学输入边界
print('\n存在缺失的字段:')  # 标明仅展示需要处置的字段
print(retail_quality.loc[retail_quality['缺失数'] > 0])  # 输出任务相关的精简质量诊断

# 原始/课堂插补双口径；不得据示例数据发布金额或客户业务结论
df_clean['amount_observed'] = df_clean['Quantity'] * df_clean['UnitPrice']
df_clean['amount_fixture_imputed'] = df_clean['Quantity'] * df_clean['UnitPrice_imputed']
quality_summary = pd.Series({'示例数据记录数': len(df_clean), '原始金额可计算行': df_clean['amount_observed'].notna().sum(), '课堂插补行': df_clean['unit_price_is_imputed'].sum(), '依据等级': '确定性课堂示例，非业务观测'})
print('=== 课堂数据质量摘要 ===')
print(quality_summary.to_string())  # 逐项输出样本边界与核心交易指标

import pandas as pd
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['Source Han Serif SC']
plt.rcParams['axes.unicode_minus'] = False

dates = pd.date_range('2024-01-01', periods=30)
prices_normal = np.array([100,101,100.5,101.2,101.0,102,101.7,102.4,102.2,103,103.1,102.9,103.2,103.4,103.6,104,103.8,104.2,104.5,104.1,104.6,104.8,105,105.2,104.9,105.3,105.5,105.4,105.8,106.0])  # 确定性课堂路径

# 创建停牌缺失
prices_suspended = prices_normal.copy()
prices_suspended[10:15] = np.nan  # 第一次停牌5天
prices_suspended[22:24] = np.nan  # 第二次停牌2天

df_stock = pd.DataFrame({
    'Date': dates,
    'Price': prices_suspended,
    'Price_Normal': prices_normal
})

print(f'缺失天数: {df_stock["Price"].isnull().sum()}')
print(f'缺失比例: {df_stock["Price"].isnull().mean():.2%}')